{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['xgb_final_nb.csv',\n",
       " 'deviceid_train.tsv',\n",
       " 'feat.csv.zip',\n",
       " '.DS_Store',\n",
       " 'thluo_train_best_feat.csv',\n",
       " 'feat.csv',\n",
       " 'xgb_feat_final_nb.csv',\n",
       " 'xgb_nb.ipynb',\n",
       " 'nurbs_feature_all.csv',\n",
       " '.ipynb_checkpoints',\n",
       " 'deviceid_test.tsv']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "import lightgbm as lgb\n",
    "from datetime import datetime,timedelta  \n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "%matplotlib inline\n",
    "\n",
    "#add\n",
    "import gc\n",
    "from sklearn import preprocessing\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "from scipy.sparse import hstack, vstack\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "# from skopt.space import Integer, Categorical, Real, Log10\n",
    "# from skopt.utils import use_named_args\n",
    "# from skopt import gp_minimize\n",
    "from gensim.models import Word2Vec, FastText\n",
    "import gensim \n",
    "import re\n",
    "import os\n",
    "path=\"./\"\n",
    "os.listdir(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_id=pd.read_csv(\"deviceid_train.tsv\",sep=\"\\t\",names=['device_id','sex','age'])\n",
    "test_id=pd.read_csv(\"deviceid_test.tsv\",sep=\"\\t\",names=['device_id'])\n",
    "all_id=pd.concat([train_id[['device_id']],test_id[['device_id']]])\n",
    "nurbs=pd.read_csv(\"nurbs_feature_all.csv\")\n",
    "nurbs.columns=[\"nurbs_\"+str(i) for i in nurbs.columns]\n",
    "all_id.index=range(len(all_id))\n",
    "nurbs['device_id']=all_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "th=pd.read_csv(\"thluo_train_best_feat.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat=pd.merge(th,nurbs,on=\"device_id\",how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat.to_csv(\"feat.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=pd.merge(train_id,feat,on=\"device_id\",how=\"left\")\n",
    "test=pd.merge(test_id,feat,on=\"device_id\",how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = [x for x in train.columns if x not in ['device_id', 'sex',\"age\",]]\n",
    "Y = train['sex'] - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/chizhu/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import xgboost as xgb\n",
    "from sklearn.metrics import auc, log_loss, roc_auc_score,f1_score,recall_score,precision_score\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "\n",
    "kf = StratifiedKFold(Y, n_folds=5, shuffle=True, random_state=1024)\n",
    "params={\n",
    "\t'booster':'gbtree',\n",
    "\t'objective': 'binary:logistic',\n",
    "#      'is_unbalance':'True',\n",
    "# \t'scale_pos_weight': 1500.0/13458.0,\n",
    "        'eval_metric': \"logloss\",\n",
    "    \n",
    "\t'gamma':0.2,#0.2 is ok\n",
    "\t'max_depth':6,\n",
    "# \t'lambda':20,\n",
    "    # \"alpha\":5,\n",
    "        'subsample':0.7,\n",
    "        'colsample_bytree':0.4 ,\n",
    "#         'min_child_weight':2.5, \n",
    "        'eta': 0.01,\n",
    "    # 'learning_rate':0.01,\n",
    "    \"silent\":1,\n",
    "\t'seed':1024,\n",
    "\t'nthread':12,\n",
    "   \n",
    "    }\n",
    "num_round = 3500\n",
    "early_stopping_rounds = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-logloss:0.691359\tval-logloss:0.691488\n",
      "Multiple eval metrics have been passed: 'val-logloss' will be used for early stopping.\n",
      "\n",
      "Will train until val-logloss hasn't improved in 100 rounds.\n",
      "[200]\ttrain-logloss:0.566693\tval-logloss:0.595722\n",
      "[400]\ttrain-logloss:0.53806\tval-logloss:0.590461\n",
      "[600]\ttrain-logloss:0.519054\tval-logloss:0.590032\n",
      "Stopping. Best iteration:\n",
      "[529]\ttrain-logloss:0.525748\tval-logloss:0.589953\n",
      "\n",
      "idx:  0\n",
      " loss: 0.59015\n",
      "[0]\ttrain-logloss:0.691215\tval-logloss:0.691369\n",
      "Multiple eval metrics have been passed: 'val-logloss' will be used for early stopping.\n",
      "\n",
      "Will train until val-logloss hasn't improved in 100 rounds.\n",
      "[200]\ttrain-logloss:0.56648\tval-logloss:0.596397\n",
      "[400]\ttrain-logloss:0.538516\tval-logloss:0.591125\n",
      "[600]\ttrain-logloss:0.51823\tval-logloss:0.590809\n",
      "Stopping. Best iteration:\n",
      "[595]\ttrain-logloss:0.518718\tval-logloss:0.590732\n",
      "\n",
      "idx:  1\n",
      " loss: 0.59099\n",
      "[0]\ttrain-logloss:0.691228\tval-logloss:0.69143\n",
      "Multiple eval metrics have been passed: 'val-logloss' will be used for early stopping.\n",
      "\n",
      "Will train until val-logloss hasn't improved in 100 rounds.\n",
      "[200]\ttrain-logloss:0.566822\tval-logloss:0.596484\n",
      "[400]\ttrain-logloss:0.538456\tval-logloss:0.591576\n",
      "[600]\ttrain-logloss:0.518551\tval-logloss:0.590934\n",
      "Stopping. Best iteration:\n",
      "[641]\ttrain-logloss:0.514957\tval-logloss:0.590818\n",
      "\n",
      "idx:  2\n",
      " loss: 0.59091\n",
      "[0]\ttrain-logloss:0.691224\tval-logloss:0.691404\n",
      "Multiple eval metrics have been passed: 'val-logloss' will be used for early stopping.\n",
      "\n",
      "Will train until val-logloss hasn't improved in 100 rounds.\n",
      "[200]\ttrain-logloss:0.565394\tval-logloss:0.598566\n",
      "[400]\ttrain-logloss:0.536792\tval-logloss:0.594022\n",
      "Stopping. Best iteration:\n",
      "[458]\ttrain-logloss:0.531227\tval-logloss:0.593837\n",
      "\n",
      "idx:  3\n",
      " loss: 0.59396\n",
      "[0]\ttrain-logloss:0.691344\tval-logloss:0.691511\n",
      "Multiple eval metrics have been passed: 'val-logloss' will be used for early stopping.\n",
      "\n",
      "Will train until val-logloss hasn't improved in 100 rounds.\n",
      "[200]\ttrain-logloss:0.566356\tval-logloss:0.595648\n",
      "[400]\ttrain-logloss:0.537421\tval-logloss:0.591302\n",
      "[600]\ttrain-logloss:0.518249\tval-logloss:0.591042\n",
      "Stopping. Best iteration:\n",
      "[525]\ttrain-logloss:0.525041\tval-logloss:0.590956\n",
      "\n",
      "idx:  4\n",
      " loss: 0.59108\n",
      "mean\n",
      "auc:       0.5914183145833928\n"
     ]
    }
   ],
   "source": [
    "aus = []\n",
    "sub1 = np.zeros((len(test), ))\n",
    "pred_oob1=np.zeros((len(train),))\n",
    "for i,(train_index,test_index) in enumerate(kf):\n",
    "  \n",
    "    tr_x = train[features].reindex(index=train_index, copy=False)\n",
    "    tr_y = Y[train_index]\n",
    "    te_x = train[features].reindex(index=test_index, copy=False)\n",
    "    te_y = Y[test_index]\n",
    "\n",
    "    # tr_y=tr_y.apply(lambda x:1 if x>0 else 0)\n",
    "    # te_y=te_y.apply(lambda x:1 if x>0 else 0)\n",
    "    d_tr = xgb.DMatrix(tr_x, label=tr_y)\n",
    "    d_te = xgb.DMatrix(te_x, label=te_y)\n",
    "    watchlist  = [(d_tr,'train'),\n",
    "    (d_te,'val')\n",
    "             ]\n",
    "    model = xgb.train(params, d_tr, num_boost_round=5500, \n",
    "                      evals=watchlist,verbose_eval=200,\n",
    "                              early_stopping_rounds=100)\n",
    "    pred = model.predict(d_te)\n",
    "    pred_oob1[test_index] =pred\n",
    "    # te_y=te_y.apply(lambda x:1 if x>0 else 0)\n",
    "    a = log_loss(te_y, pred)\n",
    "\n",
    "    sub1 += model.predict(xgb.DMatrix(test[features]))/5\n",
    "    \n",
    "\n",
    "    print (\"idx: \", i) \n",
    "    print (\" loss: %.5f\" % a)\n",
    "#     print \" gini: %.5f\" % g\n",
    "    aus.append(a)\n",
    "\n",
    "print (\"mean\")\n",
    "print (\"auc:       %s\" % (sum(aus) / 5.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_oob1 = pd.DataFrame(pred_oob1, columns=['sex2'])\n",
    "sub1 = pd.DataFrame(sub1, columns=['sex2'])\n",
    "res1=pd.concat([pred_oob1,sub1])\n",
    "res1['sex1'] = 1-res1['sex2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1012"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "####sex1\n",
    "test['sex']=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = [x for x in train.columns if x not in ['device_id',\"age\"]]\n",
    "Y = train['age'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "import xgboost as xgb\n",
    "from sklearn.metrics import auc, log_loss, roc_auc_score,f1_score,recall_score,precision_score\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "\n",
    "kf = StratifiedKFold(Y, n_folds=5, shuffle=True, random_state=1024)\n",
    "params={\n",
    "\t'booster':'gbtree',\n",
    "\t'objective': 'multi:softprob',\n",
    "#      'is_unbalance':'True',\n",
    "# \t'scale_pos_weight': 1500.0/13458.0,\n",
    "        'eval_metric': \"mlogloss\",\n",
    "    'num_class':11,\n",
    "\t'gamma':0.1,#0.2 is ok\n",
    "\t'max_depth':6,\n",
    "# \t'lambda':20,\n",
    "    # \"alpha\":5,\n",
    "        'subsample':0.7,\n",
    "        'colsample_bytree':0.4 ,\n",
    "        # 'min_child_weight':2.5, \n",
    "        'eta': 0.01,\n",
    "    # 'learning_rate':0.01,\n",
    "    \"silent\":1,\n",
    "\t'seed':1024,\n",
    "\t'nthread':12,\n",
    "   \n",
    "    }\n",
    "num_round = 3500\n",
    "early_stopping_rounds = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-mlogloss:2.39131\tval-mlogloss:2.39264\n",
      "Multiple eval metrics have been passed: 'val-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until val-mlogloss hasn't improved in 100 rounds.\n",
      "[200]\ttrain-mlogloss:1.80941\tval-mlogloss:2.00508\n",
      "[400]\ttrain-mlogloss:1.60383\tval-mlogloss:1.94\n",
      "[600]\ttrain-mlogloss:1.472\tval-mlogloss:1.9241\n",
      "[800]\ttrain-mlogloss:1.36689\tval-mlogloss:1.92024\n",
      "[1000]\ttrain-mlogloss:1.273\tval-mlogloss:1.91999\n",
      "Stopping. Best iteration:\n",
      "[918]\ttrain-mlogloss:1.31045\tval-mlogloss:1.91983\n",
      "\n",
      "idx:  0\n",
      " loss: 1.91985\n",
      "[0]\ttrain-mlogloss:2.39114\tval-mlogloss:2.39277\n",
      "Multiple eval metrics have been passed: 'val-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until val-mlogloss hasn't improved in 100 rounds.\n",
      "[200]\ttrain-mlogloss:1.8078\tval-mlogloss:2.0115\n",
      "[400]\ttrain-mlogloss:1.60116\tval-mlogloss:1.9457\n",
      "[600]\ttrain-mlogloss:1.46953\tval-mlogloss:1.93011\n",
      "[800]\ttrain-mlogloss:1.36553\tval-mlogloss:1.92647\n",
      "Stopping. Best iteration:\n",
      "[825]\ttrain-mlogloss:1.35318\tval-mlogloss:1.92626\n",
      "\n",
      "idx:  1\n",
      " loss: 1.92627\n",
      "[0]\ttrain-mlogloss:2.39122\tval-mlogloss:2.3928\n",
      "Multiple eval metrics have been passed: 'val-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until val-mlogloss hasn't improved in 100 rounds.\n",
      "[200]\ttrain-mlogloss:1.8065\tval-mlogloss:2.01298\n",
      "[400]\ttrain-mlogloss:1.60091\tval-mlogloss:1.94872\n",
      "[600]\ttrain-mlogloss:1.4685\tval-mlogloss:1.93313\n",
      "[800]\ttrain-mlogloss:1.36383\tval-mlogloss:1.92927\n",
      "Stopping. Best iteration:\n",
      "[899]\ttrain-mlogloss:1.3168\tval-mlogloss:1.92877\n",
      "\n",
      "idx:  2\n",
      " loss: 1.92879\n",
      "[0]\ttrain-mlogloss:2.39105\tval-mlogloss:2.39257\n",
      "Multiple eval metrics have been passed: 'val-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until val-mlogloss hasn't improved in 100 rounds.\n",
      "[200]\ttrain-mlogloss:1.80767\tval-mlogloss:2.01163\n",
      "[400]\ttrain-mlogloss:1.6018\tval-mlogloss:1.94808\n",
      "[600]\ttrain-mlogloss:1.47112\tval-mlogloss:1.93282\n",
      "[800]\ttrain-mlogloss:1.36743\tval-mlogloss:1.92918\n",
      "[1000]\ttrain-mlogloss:1.27495\tval-mlogloss:1.92918\n",
      "Stopping. Best iteration:\n",
      "[953]\ttrain-mlogloss:1.29641\tval-mlogloss:1.92904\n",
      "\n",
      "idx:  3\n",
      " loss: 1.92906\n",
      "[0]\ttrain-mlogloss:2.39143\tval-mlogloss:2.39284\n",
      "Multiple eval metrics have been passed: 'val-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until val-mlogloss hasn't improved in 100 rounds.\n",
      "[200]\ttrain-mlogloss:1.81054\tval-mlogloss:2.00446\n",
      "[400]\ttrain-mlogloss:1.6046\tval-mlogloss:1.93723\n",
      "[600]\ttrain-mlogloss:1.47282\tval-mlogloss:1.92063\n",
      "[800]\ttrain-mlogloss:1.36819\tval-mlogloss:1.91661\n",
      "[1000]\ttrain-mlogloss:1.27547\tval-mlogloss:1.91579\n",
      "Stopping. Best iteration:\n",
      "[1014]\ttrain-mlogloss:1.26898\tval-mlogloss:1.91575\n",
      "\n",
      "idx:  4\n",
      " loss: 1.91579\n",
      "mean\n",
      "auc:       1.923953299949125\n"
     ]
    }
   ],
   "source": [
    "aus = []\n",
    "sub2 = np.zeros((len(test),11 ))\n",
    "pred_oob2=np.zeros((len(train),11))\n",
    "models=[]\n",
    "iters=[]\n",
    "for i,(train_index,test_index) in enumerate(kf):\n",
    "  \n",
    "    tr_x = train[features].reindex(index=train_index, copy=False)\n",
    "    tr_y = Y[train_index]\n",
    "    te_x = train[features].reindex(index=test_index, copy=False)\n",
    "    te_y = Y[test_index]\n",
    "\n",
    "    # tr_y=tr_y.apply(lambda x:1 if x>0 else 0)\n",
    "    # te_y=te_y.apply(lambda x:1 if x>0 else 0)\n",
    "    d_tr = xgb.DMatrix(tr_x, label=tr_y)\n",
    "    d_te = xgb.DMatrix(te_x, label=te_y)\n",
    "    watchlist  = [(d_tr,'train'),\n",
    "    (d_te,'val')\n",
    "             ]\n",
    "    model = xgb.train(params, d_tr, num_boost_round=5500, \n",
    "                      evals=watchlist,verbose_eval=200,\n",
    "                              early_stopping_rounds=100)\n",
    "    models.append(model)\n",
    "    iters.append(model.best_iteration)\n",
    "    pred = model.predict(d_te,ntree_limit=model.best_iteration)\n",
    "    pred_oob2[test_index] =pred\n",
    "    # te_y=te_y.apply(lambda x:1 if x>0 else 0)\n",
    "    a = log_loss(te_y, pred)\n",
    "\n",
    "    sub2 += model.predict(xgb.DMatrix(test[features]),ntree_limit=model.best_iteration)/5\n",
    "    \n",
    "\n",
    "    print (\"idx: \", i) \n",
    "    print (\" loss: %.5f\" % a)\n",
    "#     print \" gini: %.5f\" % g\n",
    "    aus.append(a)\n",
    "\n",
    "print (\"mean\")\n",
    "print (\"auc:       %s\" % (sum(aus) / 5.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "res2_1=np.vstack((pred_oob2,sub2))\n",
    "res2_1 = pd.DataFrame(res2_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "###sex2\n",
    "test['sex']=2\n",
    "features = [x for x in train.columns if x not in ['device_id',\"age\",\"label\",\"app\"]]\n",
    "Y = train['age'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "aus = []\n",
    "sub2 = np.zeros((len(test),11 ))\n",
    "for model,it in zip(models,iters):\n",
    "    sub2 += model.predict(xgb.DMatrix(test[features]),ntree_limit=it)/5\n",
    "res2_2=np.vstack((pred_oob2,sub2))\n",
    "res2_2 = pd.DataFrame(res2_2) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "res1.index=range(len(res1))\n",
    "res2_1.index=range(len(res2_1))\n",
    "res2_2.index=range(len(res2_2))\n",
    "final_1=res2_1.copy()\n",
    "final_2=res2_2.copy()\n",
    "for i in range(11):\n",
    "    final_1[i]=res1['sex1']*res2_1[i]\n",
    "    final_2[i]=res1['sex2']*res2_2[i]\n",
    "id_list=pd.concat([train[['device_id']],test[['device_id']]])\n",
    "final=id_list\n",
    "final.index=range(len(final))\n",
    "final.columns= ['DeviceID']\n",
    "final_pred = pd.concat([final_1,final_2],1)\n",
    "final=pd.concat([final,final_pred],1)\n",
    "final.columns = ['DeviceID', '1-0', '1-1', '1-2', '1-3', '1-4', '1-5', '1-6', \n",
    "         '1-7','1-8', '1-9', '1-10', '2-0', '2-1', '2-2', '2-3', '2-4', \n",
    "         '2-5', '2-6', '2-7', '2-8', '2-9', '2-10']\n",
    "\n",
    "final.to_csv('xgb_feat_final_nb.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "test['DeviceID']=test['device_id']\n",
    "sub=pd.merge(test[['DeviceID']],final,on=\"DeviceID\",how=\"left\")\n",
    "sub.to_csv(\"xgb_final_nb.csv\",index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
